#!/usr/bin/env python
# -*- coding:utf-8 -*-
__author__ = 'wshu'
__version__ = '1.0'
"""
    ***********************************
    *  @filename : pipelines.py
    *  @Author : wshu
    *  @CodeDate : 2020/3/26 11:06
    *  @Software : PyCharm
    ***********************************
"""
import pymysql
from scrapy.exceptions import NotConfigured


class HfbotV1Pipeline(object):
    """
        :except 爬虫存储到Mysql功能对应Settings.py 配置信息.
        1. 加载mysql配置
        2. 存储数据
    """
    def __init__(self, host, port, db, user, passwd):
        """ mysql加载"""
        self.host = host
        self.port = port
        self.db = db
        self.user = user
        self.passwd = passwd

    @classmethod
    def from_crawler(cls, crawler):
        """
            加载配置文件，读取配置信息
        """
        dbconf = crawler.settings.getdict("MYSQL_CONF")
        if not dbconf:
            raise NotConfigured('read settings file failed')
        host = dbconf.get('MYSQL_HOST')
        port = dbconf.get('MYSQL_PORT')
        db = dbconf.get('MYSQL_DB')
        user = dbconf.get('MYSQL_USER')
        passwd = dbconf.get('MYSQL_PWD')
        return cls(host=host, port=port, db=db, user=user, passwd=passwd)

    def open_spider(self, spider):
        """
            开启爬虫连接数据库
        """
        try:
            self.connect = pymysql.connect(
                host=self.host,
                port=self.port,
                database=self.db,
                user=self.user,
                passwd=self.passwd,
                charset='utf8',
                cursorclass=pymysql.cursors.DictCursor,
                use_unicode=True)
            self.cursor = self.connect.cursor()
        except Exception as err:
            print('连接MySQL错误: ->{}'.format(err))
            raise ConnectionError('Error: <Connection database exception>')


    def process_item(self, item, spider):
        """
            存储过程
        """
        return item


    def close_spider(self, spider):
        """
            爬虫结束时释放数据库资源
        """
        self.cursor.close()
        self.connect.close()